{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Classical RL for Portfolio Management on Foreign Exchange\n",
    "This tutorial is to demonstrate an example of using classical reinforcement learning to do portfolio management on foreign exchange\n",
    "## Set up Experinment Environment"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Collecting package metadata (current_repodata.json): \\ ^C\n",
      "failed\n",
      "\n",
      "CondaError: KeyboardInterrupt\n",
      "\n"
     ]
    }
   ],
   "source": [
    "from IPython.display import clear_output\n",
    "import argparse\n",
    "import sys\n",
    "import numpy as np\n",
    "import torch\n",
    "from torch import nn\n",
    "import yaml\n",
    "import os\n",
    "import pandas as pd\n",
    "module_path = os.path.abspath(os.path.join('..'))\n",
    "sys.path.append(module_path)\n",
    "requirements_path=module_path+\"/requirements.txt\"\n",
    "print(requirements_path)\n",
    "command=\"pip install -r \"+requirements_path\n",
    "os.system(command)\n",
    "clear_output(wait=True)\n",
    "! conda install pytorch torchvision torchaudio cudatoolkit=11.3 -c pytorch\n",
    "clear_output(wait=True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Download and Preprocess the Data\n",
    "The dataconfig part in TradeMaster is shared by all other parts, it is worth noticing that for algorithms trading, only the dataset BTC is supported.\n",
    "\n",
    "The following code will help to download the data in the folder [.data/data/exchange](https://github.com/qinmoelei/TradeMaster_reframe/tree/master/tutorial/data/data/exchange), where 4 files could be found: the whole data and train, valid and test data which will be used when we try to construct the RL environment for the agent"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/sunshuo/qml/TradeMaster/data/preprocess.py:19: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "  (temp_indicator.close.rolling(2).sum() - temp_indicator.close)) - 1\n",
      "/home/sunshuo/qml/TradeMaster/data/preprocess.py:21: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "  5) / temp_indicator.adjcp - 1\n",
      "/home/sunshuo/qml/TradeMaster/data/preprocess.py:23: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "  10) / temp_indicator.adjcp - 1\n",
      "/home/sunshuo/qml/TradeMaster/data/preprocess.py:25: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "  15) / temp_indicator.adjcp - 1\n",
      "/home/sunshuo/qml/TradeMaster/data/preprocess.py:27: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "  20) / temp_indicator.adjcp - 1\n",
      "/home/sunshuo/qml/TradeMaster/data/preprocess.py:29: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "  25) / temp_indicator.adjcp - 1\n",
      "/home/sunshuo/qml/TradeMaster/data/preprocess.py:31: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "  30) / temp_indicator.adjcp - 1\n"
     ]
    }
   ],
   "source": [
    "from data.download_data import Dataconfig\n",
    "parser = argparse.ArgumentParser()\n",
    "\n",
    "parser.add_argument(\"--data_path\",\n",
    "                    type=str,\n",
    "                    default=\"./data/data/\",\n",
    "                    help=\"the path for storing the downloaded data\")\n",
    "#where we store the dataset\n",
    "parser.add_argument(\n",
    "    \"--output_config_path\",\n",
    "    type=str,\n",
    "    default=\"./config/output_config/data\",\n",
    "    help=\"the path for storing the generated config file for data\")\n",
    "# where we store the config file\n",
    "parser.add_argument(\n",
    "    \"--dataset\",\n",
    "    choices=[\"exchange\",\"dj30\",\"sz50\",\"crypto\"],\n",
    "    default=\"exchange\",\n",
    "    help=\"the name of the dataset\",\n",
    ")\n",
    "parser.add_argument(\"--split_proportion\",\n",
    "                    type=list,\n",
    "                    default=[0.8, 0.1, 0.1],\n",
    "                    help=\"the split proportion for train, valid and test\")\n",
    "parser.add_argument(\n",
    "    \"--generate_config\",\n",
    "    type=bool,\n",
    "    default=False,\n",
    "    help=\n",
    "    \"determine whether to generate a yaml file to memorize the train valid and test'data's dict\"\n",
    ")\n",
    "parser.add_argument(\n",
    "    \"--input_config\",\n",
    "    type=bool,\n",
    "    default=False,\n",
    "    help=\n",
    "    \"determine whether to use a yaml file as the overall input of the Dataconfig, this is needed when have other format of dataset\"\n",
    ")\n",
    "\n",
    "parser.add_argument(\n",
    "    \"--input_config_path\",\n",
    "    type=str,\n",
    "    default=\"config/input_config/data/custom.yml\",\n",
    "    help=\n",
    "    \"determine the location of a yaml file used to initialize the Dataconfig Class\"\n",
    ")\n",
    "args = parser.parse_args(args=[])\n",
    "a = Dataconfig(args)\n",
    "clear_output(wait=True)\n",
    "clear_output(wait=True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "The preprocessed data follows the following structure:\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>date</th>\n",
       "      <th>close</th>\n",
       "      <th>tic</th>\n",
       "      <th>open</th>\n",
       "      <th>high</th>\n",
       "      <th>low</th>\n",
       "      <th>adjcp</th>\n",
       "      <th>zopen</th>\n",
       "      <th>zhigh</th>\n",
       "      <th>zlow</th>\n",
       "      <th>zadjcp</th>\n",
       "      <th>zclose</th>\n",
       "      <th>zd_5</th>\n",
       "      <th>zd_10</th>\n",
       "      <th>zd_15</th>\n",
       "      <th>zd_20</th>\n",
       "      <th>zd_25</th>\n",
       "      <th>zd_30</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>2000-01-03</td>\n",
       "      <td>0.659109</td>\n",
       "      <td>AUSTRALIAN DOLLAR</td>\n",
       "      <td>0.659109</td>\n",
       "      <td>0.659109</td>\n",
       "      <td>0.659109</td>\n",
       "      <td>0.659109</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>-0.004397</td>\n",
       "      <td>0.001568</td>\n",
       "      <td>-0.011881</td>\n",
       "      <td>0.00816</td>\n",
       "      <td>0.027933</td>\n",
       "      <td>0.02804</td>\n",
       "      <td>0.032544</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>2000-01-04</td>\n",
       "      <td>0.656211</td>\n",
       "      <td>AUSTRALIAN DOLLAR</td>\n",
       "      <td>0.656211</td>\n",
       "      <td>0.656211</td>\n",
       "      <td>0.656211</td>\n",
       "      <td>0.656211</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>-0.004397</td>\n",
       "      <td>0.001568</td>\n",
       "      <td>-0.011881</td>\n",
       "      <td>0.00816</td>\n",
       "      <td>0.027933</td>\n",
       "      <td>0.02804</td>\n",
       "      <td>0.032544</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>2000-01-05</td>\n",
       "      <td>0.655008</td>\n",
       "      <td>AUSTRALIAN DOLLAR</td>\n",
       "      <td>0.655008</td>\n",
       "      <td>0.655008</td>\n",
       "      <td>0.655008</td>\n",
       "      <td>0.655008</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>-0.001834</td>\n",
       "      <td>0.001568</td>\n",
       "      <td>-0.011881</td>\n",
       "      <td>0.00816</td>\n",
       "      <td>0.027933</td>\n",
       "      <td>0.02804</td>\n",
       "      <td>0.032544</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>2000-01-06</td>\n",
       "      <td>0.653979</td>\n",
       "      <td>AUSTRALIAN DOLLAR</td>\n",
       "      <td>0.653979</td>\n",
       "      <td>0.653979</td>\n",
       "      <td>0.653979</td>\n",
       "      <td>0.653979</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>-0.001570</td>\n",
       "      <td>0.001568</td>\n",
       "      <td>-0.011881</td>\n",
       "      <td>0.00816</td>\n",
       "      <td>0.027933</td>\n",
       "      <td>0.02804</td>\n",
       "      <td>0.032544</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>2000-01-07</td>\n",
       "      <td>0.654793</td>\n",
       "      <td>AUSTRALIAN DOLLAR</td>\n",
       "      <td>0.654793</td>\n",
       "      <td>0.654793</td>\n",
       "      <td>0.654793</td>\n",
       "      <td>0.654793</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.001244</td>\n",
       "      <td>0.001568</td>\n",
       "      <td>-0.011881</td>\n",
       "      <td>0.00816</td>\n",
       "      <td>0.027933</td>\n",
       "      <td>0.02804</td>\n",
       "      <td>0.032544</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "         date     close                tic      open      high       low  \\\n",
       "0  2000-01-03  0.659109  AUSTRALIAN DOLLAR  0.659109  0.659109  0.659109   \n",
       "1  2000-01-04  0.656211  AUSTRALIAN DOLLAR  0.656211  0.656211  0.656211   \n",
       "2  2000-01-05  0.655008  AUSTRALIAN DOLLAR  0.655008  0.655008  0.655008   \n",
       "3  2000-01-06  0.653979  AUSTRALIAN DOLLAR  0.653979  0.653979  0.653979   \n",
       "4  2000-01-07  0.654793  AUSTRALIAN DOLLAR  0.654793  0.654793  0.654793   \n",
       "\n",
       "      adjcp  zopen  zhigh  zlow  zadjcp    zclose      zd_5     zd_10  \\\n",
       "0  0.659109    0.0    0.0   0.0     0.0 -0.004397  0.001568 -0.011881   \n",
       "1  0.656211    0.0    0.0   0.0     0.0 -0.004397  0.001568 -0.011881   \n",
       "2  0.655008    0.0    0.0   0.0     0.0 -0.001834  0.001568 -0.011881   \n",
       "3  0.653979    0.0    0.0   0.0     0.0 -0.001570  0.001568 -0.011881   \n",
       "4  0.654793    0.0    0.0   0.0     0.0  0.001244  0.001568 -0.011881   \n",
       "\n",
       "     zd_15     zd_20    zd_25     zd_30  \n",
       "0  0.00816  0.027933  0.02804  0.032544  \n",
       "1  0.00816  0.027933  0.02804  0.032544  \n",
       "2  0.00816  0.027933  0.02804  0.032544  \n",
       "3  0.00816  0.027933  0.02804  0.032544  \n",
       "4  0.00816  0.027933  0.02804  0.032544  "
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data=pd.read_csv(\"data/data/exchange/exchange.csv\",index_col=0)\n",
    "data.head(5)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "the index column are corresponding to date one by one and in the algorithm trading case, there are multiple tickers. \n",
    "\n",
    "Besides OHLC, the data also has some normalized features"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## RL Enviornment Construction, Agent Training, Model Picking and Testing\n",
    "For the simplicity, we use the yaml file to store the configuration for the RL environment construction, which can be found [here](https://github.com/qinmoelei/TradeMaster_reframe/tree/master/tutorial/config/input_config/env/AT/DeepScalper)\n",
    "but first, let us import the package we might use"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2022-08-20 23:55:00,169\tINFO worker.py:973 -- Calling ray.init() again after it has already been called.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<class 'ray.rllib.agents.a3c.a2c.A2CTrainer'>\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\u001b[2m\u001b[36m(RolloutWorker pid=3866172)\u001b[0m 2022-08-20 23:55:04,933\tWARNING env.py:136 -- Your env doesn't have a .spec.max_episode_steps attribute. This is fine if you have set 'horizon' in your config dictionary, or `soft_horizon`. However, if you haven't, 'horizon' will default to infinity, and your environment will not be reset.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "=================================\n",
      "the profit margin is -2.657352222665521 %\n",
      "the sharpe ratio is -0.27945782731776925\n",
      "the Volatility is 0.0037477005240960917\n",
      "the max drawdown is 0.11981696588375827\n",
      "the Calmar Ratio is -0.19545547586237488\n",
      "the Sortino Ratio is -0.40876710691513896\n",
      "=================================\n",
      "\u001b[2m\u001b[36m(RolloutWorker pid=3866172)\u001b[0m =================================\n",
      "\u001b[2m\u001b[36m(RolloutWorker pid=3866172)\u001b[0m the profit margin is -51.61219528306229 %\n",
      "\u001b[2m\u001b[36m(RolloutWorker pid=3866172)\u001b[0m the sharpe ratio is -2.634938863151262\n",
      "\u001b[2m\u001b[36m(RolloutWorker pid=3866172)\u001b[0m the Volatility is 0.004140504909855382\n",
      "\u001b[2m\u001b[36m(RolloutWorker pid=3866172)\u001b[0m the max drawdown is 0.5182227113652991\n",
      "\u001b[2m\u001b[36m(RolloutWorker pid=3866172)\u001b[0m the Calmar Ratio is -1.334148759697776\n",
      "\u001b[2m\u001b[36m(RolloutWorker pid=3866172)\u001b[0m the Sortino Ratio is -3.4944424536508794\n",
      "\u001b[2m\u001b[36m(RolloutWorker pid=3866172)\u001b[0m =================================\n",
      "=================================\n",
      "the profit margin is -5.159756755010092 %\n",
      "the sharpe ratio is -0.6094276604532715\n",
      "the Volatility is 0.003643611694495803\n",
      "the max drawdown is 0.08546099170063744\n",
      "the Calmar Ratio is -0.580993566389602\n",
      "the Sortino Ratio is -0.8519870108015014\n",
      "=================================\n",
      "\u001b[2m\u001b[36m(RolloutWorker pid=3866172)\u001b[0m =================================\n",
      "\u001b[2m\u001b[36m(RolloutWorker pid=3866172)\u001b[0m the profit margin is -48.886371168777785 %\n",
      "\u001b[2m\u001b[36m(RolloutWorker pid=3866172)\u001b[0m the sharpe ratio is -2.4786527602013857\n",
      "\u001b[2m\u001b[36m(RolloutWorker pid=3866172)\u001b[0m the Volatility is 0.004061361087707681\n",
      "\u001b[2m\u001b[36m(RolloutWorker pid=3866172)\u001b[0m the max drawdown is 0.4919419173404109\n",
      "\u001b[2m\u001b[36m(RolloutWorker pid=3866172)\u001b[0m the Calmar Ratio is -1.2967919797479397\n",
      "\u001b[2m\u001b[36m(RolloutWorker pid=3866172)\u001b[0m the Sortino Ratio is -3.3826964171131744\n",
      "\u001b[2m\u001b[36m(RolloutWorker pid=3866172)\u001b[0m =================================\n",
      "=================================\n",
      "the profit margin is -0.1868654496308686 %\n",
      "the sharpe ratio is 0.021027897658670157\n",
      "the Volatility is 0.003835646776036807\n",
      "the max drawdown is 0.07430196803383515\n",
      "the Calmar Ratio is 0.024272759120050363\n",
      "the Sortino Ratio is 0.032230271384267595\n",
      "=================================\n",
      "=================================\n",
      "the profit margin is -1.4281926056985972 %\n",
      "the sharpe ratio is -0.1281394340919933\n",
      "the Volatility is 0.0037784555494351185\n",
      "the max drawdown is 0.08238132179528922\n",
      "the Calmar Ratio is -0.13141754969305142\n",
      "the Sortino Ratio is -0.22020564830025502\n",
      "=================================\n",
      "\u001b[2m\u001b[36m(RolloutWorker pid=3866172)\u001b[0m =================================\n",
      "\u001b[2m\u001b[36m(RolloutWorker pid=3866172)\u001b[0m the profit margin is -41.75836131360834 %\n",
      "\u001b[2m\u001b[36m(RolloutWorker pid=3866172)\u001b[0m the sharpe ratio is -2.0098137311224087\n",
      "\u001b[2m\u001b[36m(RolloutWorker pid=3866172)\u001b[0m the Volatility is 0.003992528879206729\n",
      "\u001b[2m\u001b[36m(RolloutWorker pid=3866172)\u001b[0m the max drawdown is 0.4260967783291352\n",
      "\u001b[2m\u001b[36m(RolloutWorker pid=3866172)\u001b[0m the Calmar Ratio is -1.193417705502005\n",
      "\u001b[2m\u001b[36m(RolloutWorker pid=3866172)\u001b[0m the Sortino Ratio is -2.773203152996862\n",
      "\u001b[2m\u001b[36m(RolloutWorker pid=3866172)\u001b[0m =================================\n",
      "=================================\n",
      "the profit margin is 0.6118150257782062 %\n",
      "the sharpe ratio is 0.11399594879682774\n",
      "the Volatility is 0.003833357584868862\n",
      "the max drawdown is 0.07903078616124151\n",
      "the Calmar Ratio is 0.12363955999210606\n",
      "the Sortino Ratio is 0.16595036281124934\n",
      "=================================\n",
      "\u001b[2m\u001b[36m(RolloutWorker pid=3866172)\u001b[0m =================================\n",
      "\u001b[2m\u001b[36m(RolloutWorker pid=3866172)\u001b[0m the profit margin is -37.126052087339744 %\n",
      "\u001b[2m\u001b[36m(RolloutWorker pid=3866172)\u001b[0m the sharpe ratio is -1.6998779031209355\n",
      "\u001b[2m\u001b[36m(RolloutWorker pid=3866172)\u001b[0m the Volatility is 0.0040079274016280985\n",
      "\u001b[2m\u001b[36m(RolloutWorker pid=3866172)\u001b[0m the max drawdown is 0.3989653298117346\n",
      "\u001b[2m\u001b[36m(RolloutWorker pid=3866172)\u001b[0m the Calmar Ratio is -1.0821794179300284\n",
      "\u001b[2m\u001b[36m(RolloutWorker pid=3866172)\u001b[0m the Sortino Ratio is -2.3560528642688636\n",
      "\u001b[2m\u001b[36m(RolloutWorker pid=3866172)\u001b[0m =================================\n",
      "=================================\n",
      "the profit margin is -3.654923803216714 %\n",
      "the sharpe ratio is -0.3840710294429422\n",
      "the Volatility is 0.003893712378717129\n",
      "the max drawdown is 0.08387963249772774\n",
      "the Calmar Ratio is -0.3986611364651542\n",
      "the Sortino Ratio is -0.5280563262145817\n",
      "=================================\n",
      "=================================\n",
      "the profit margin is -2.9759132409266753 %\n",
      "the sharpe ratio is -0.31920628823600283\n",
      "the Volatility is 0.0037423006053932885\n",
      "the max drawdown is 0.09738231619515884\n",
      "the Calmar Ratio is -0.27429318057077506\n",
      "the Sortino Ratio is -0.4862876697292982\n",
      "=================================\n",
      "\u001b[2m\u001b[36m(RolloutWorker pid=3866172)\u001b[0m =================================\n",
      "\u001b[2m\u001b[36m(RolloutWorker pid=3866172)\u001b[0m the profit margin is -44.098006609404806 %\n",
      "\u001b[2m\u001b[36m(RolloutWorker pid=3866172)\u001b[0m the sharpe ratio is -2.105172859849299\n",
      "\u001b[2m\u001b[36m(RolloutWorker pid=3866172)\u001b[0m the Volatility is 0.004105107838886465\n",
      "\u001b[2m\u001b[36m(RolloutWorker pid=3866172)\u001b[0m the max drawdown is 0.44588721705341317\n",
      "\u001b[2m\u001b[36m(RolloutWorker pid=3866172)\u001b[0m the Calmar Ratio is -1.2282426185905755\n",
      "\u001b[2m\u001b[36m(RolloutWorker pid=3866172)\u001b[0m the Sortino Ratio is -2.806643890555294\n",
      "\u001b[2m\u001b[36m(RolloutWorker pid=3866172)\u001b[0m =================================\n",
      "=================================\n",
      "the profit margin is -0.681370296199113 %\n",
      "the sharpe ratio is -0.03577620505361049\n",
      "the Volatility is 0.0038683979699339517\n",
      "the max drawdown is 0.0966218745260016\n",
      "the Calmar Ratio is -0.032028379157399714\n",
      "the Sortino Ratio is -0.050119333870781234\n",
      "=================================\n",
      "\u001b[2m\u001b[36m(RolloutWorker pid=3866172)\u001b[0m =================================\n",
      "\u001b[2m\u001b[36m(RolloutWorker pid=3866172)\u001b[0m the profit margin is -56.96620410885925 %\n",
      "\u001b[2m\u001b[36m(RolloutWorker pid=3866172)\u001b[0m the sharpe ratio is -3.208822516833007\n",
      "\u001b[2m\u001b[36m(RolloutWorker pid=3866172)\u001b[0m the Volatility is 0.003988650687003602\n",
      "\u001b[2m\u001b[36m(RolloutWorker pid=3866172)\u001b[0m the max drawdown is 0.5827124370178582\n",
      "\u001b[2m\u001b[36m(RolloutWorker pid=3866172)\u001b[0m the Calmar Ratio is -1.391919967540197\n",
      "\u001b[2m\u001b[36m(RolloutWorker pid=3866172)\u001b[0m the Sortino Ratio is -4.201985081304954\n",
      "\u001b[2m\u001b[36m(RolloutWorker pid=3866172)\u001b[0m =================================\n",
      "=================================\n",
      "the profit margin is 0.6843740462739722 %\n",
      "the sharpe ratio is 0.12093245524484547\n",
      "the Volatility is 0.003998424029183239\n",
      "the max drawdown is 0.08237078445273231\n",
      "the Calmar Ratio is 0.13126336070980107\n",
      "the Sortino Ratio is 0.17851991504801812\n",
      "=================================\n",
      "\u001b[2m\u001b[36m(RolloutWorker pid=3866172)\u001b[0m =================================\n",
      "\u001b[2m\u001b[36m(RolloutWorker pid=3866172)\u001b[0m the profit margin is -52.529918001402606 %\n",
      "\u001b[2m\u001b[36m(RolloutWorker pid=3866172)\u001b[0m the sharpe ratio is -2.780841141342315\n",
      "\u001b[2m\u001b[36m(RolloutWorker pid=3866172)\u001b[0m the Volatility is 0.004041108416289723\n",
      "\u001b[2m\u001b[36m(RolloutWorker pid=3866172)\u001b[0m the max drawdown is 0.5278521392420458\n",
      "\u001b[2m\u001b[36m(RolloutWorker pid=3866172)\u001b[0m the Calmar Ratio is -1.3491531388932578\n",
      "\u001b[2m\u001b[36m(RolloutWorker pid=3866172)\u001b[0m the Sortino Ratio is -3.6279121401771084\n",
      "\u001b[2m\u001b[36m(RolloutWorker pid=3866172)\u001b[0m =================================\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2022-08-20 23:57:15,083\tINFO trainable.py:589 -- Restored on 172.21.100.188 from checkpoint: /home/sunshuo/ray_results/A2CTrainer_portfolio_2022-08-20_23-55-00ypvuw33r/checkpoint_000009/checkpoint-9\n",
      "2022-08-20 23:57:15,085\tINFO trainable.py:597 -- Current state after restoring: {'_iteration': 9, '_timesteps_total': None, '_time_total': 94.79972052574158, '_episodes_total': 6}\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "=================================\n",
      "the profit margin is -6.830574814325696 %\n",
      "the sharpe ratio is -0.7348759446989664\n",
      "the Volatility is 0.00405463608225716\n",
      "the max drawdown is 0.14311833704354762\n",
      "the Calmar Ratio is -0.4655385324381294\n",
      "the Sortino Ratio is -0.9377936831335998\n",
      "=================================\n",
      "=================================\n",
      "the profit margin is -14.081375922478955 %\n",
      "the sharpe ratio is -2.3356888920887653\n",
      "the Volatility is 0.0028718863556721807\n",
      "the max drawdown is 0.1821742235412546\n",
      "the Calmar Ratio is -0.8216937618087252\n",
      "the Sortino Ratio is -3.342926920783735\n",
      "=================================\n"
     ]
    }
   ],
   "source": [
    "from agent.ClassicRL.SOTA import trader,env_creator,load_yaml,select_algorithms\n",
    "from env.PM.portfolio_management import TradingEnv\n",
    "parser = argparse.ArgumentParser()\n",
    "\n",
    "parser.add_argument(\n",
    "    \"--env_name\",\n",
    "    choices=[\"portfolio\"],\n",
    "    default=\"portfolio\",\n",
    "    help=\"the name of TradingEnv \",\n",
    ")\n",
    "parser.add_argument(\n",
    "    \"--dict_trained_model\",\n",
    "    default=\"result/PM/SOTA/trained_model/\",\n",
    "    help=\"the dict of the trained model \",\n",
    ")\n",
    "\n",
    "parser.add_argument(\n",
    "    \"--train_env_config_dict\",\n",
    "    default=\"config/input_config/env/portfolio/portfolio/train.yml\",\n",
    "    help=\"the dict of the train config of TradingEnv \",\n",
    ")\n",
    "\n",
    "parser.add_argument(\n",
    "    \"--valid_env_config_dict\",\n",
    "    default=\"config/input_config/env/portfolio/portfolio/valid.yml\",\n",
    "    help=\"the dict of the valid config of TradingEnv \",\n",
    ")\n",
    "\n",
    "parser.add_argument(\n",
    "    \"--test_env_config_dict\",\n",
    "    default=\"config/input_config/env/portfolio/portfolio/test.yml\",\n",
    "    help=\"the dict of the test config of TradingEnv \",\n",
    ")\n",
    "\n",
    "parser.add_argument(\n",
    "    \"--name_of_algorithms\",\n",
    "    choices=[\"PPO\", \"A2C\", \"SAC\", \"TD3\", \"PG\", \"DDPG\"],\n",
    "    type=str,\n",
    "    default=\"A2C\",\n",
    "    help=\"name_of_algorithms \",\n",
    ")\n",
    "parser.add_argument(\n",
    "    \"--num_epochs\",\n",
    "    type=int,\n",
    "    default=10,\n",
    "    help=\"the number of training epoch\",\n",
    ")\n",
    "\n",
    "parser.add_argument(\n",
    "    \"--random_seed\",\n",
    "    type=int,\n",
    "    default=12345,\n",
    "    help=\"the number of training epoch\",\n",
    ")\n",
    "\n",
    "parser.add_argument(\n",
    "    \"--model_config_dict\",\n",
    "    type=str,\n",
    "    default=\"config/input_config/agent/SOTA/A2C.yml\",\n",
    "    help=\"the dict of the model_config file\",\n",
    ")\n",
    "\n",
    "parser.add_argument(\n",
    "    \"--result_dict\",\n",
    "    type=str,\n",
    "    default=\"result/PM/SOTA/test_result/\",\n",
    "    help=\"the dict of the result of the test\",\n",
    ")\n",
    "\n",
    "\n",
    "args = parser.parse_args(args=[])\n",
    "a = trader(args)\n",
    "a.train_with_valid()\n",
    "a.test()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3.7.13 ('TradeMaster')",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.13"
  },
  "orig_nbformat": 4,
  "vscode": {
   "interpreter": {
    "hash": "a1f97403911abd3f02553c8f2b0c54537fddc7efadd9f5d3e31784db6e40c347"
   }
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
